{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import tree\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor\n",
    "from sklearn.model_selection import cross_val_score,train_test_split,GridSearchCV,KFold\n",
    "from xgboost import XGBRegressor,XGBRFRegressor\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1084, 38), (1084,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#特征工程\n",
    "#train_train1是训练集特征，train_feature是测试集特征 test_target是测试结果\n",
    "train_data = pd.read_csv('zhengqi_train.txt',sep='\\t')\n",
    "test_data = pd.read_csv('zhengqi_test.txt',sep='\\t')\n",
    "train_data0 = train_data[train_data['V9']>-7.5] \n",
    "train_data0 = train_data0[train_data0['V36']<4.5]\n",
    "#pre_feature = ['V0','V1','V2','V3','V4','V6','V7','V8','V10','V12','V13','V15','V16','V18','V19','V20','V21','V23','V24','V25','V26','V27',\n",
    "#               'V29','V30','V31','V32','V33','V34','V35','V36','V37']\n",
    "pre_feature = ['V0','V1','V2','V3','V6','V7','V8','V10','V13','V15','V18','V19','V24','V29','V30','V31','V36','V37']\n",
    "train_train = train_data0.sample(n=1800)\n",
    "temp_data = train_data0\n",
    "index = list(train_train.index)\n",
    "train_test = train_data0.drop(index)\n",
    "train_trainf = train_train[pre_feature]\n",
    "train_traint = train_train['target']\n",
    "test_feature = train_test.iloc[:,:-1]\n",
    "test_target = train_test.iloc[:,-1]\n",
    "test_feature.shape,test_target.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V0</th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V9</th>\n",
       "      <th>...</th>\n",
       "      <th>V29</th>\n",
       "      <th>V30</th>\n",
       "      <th>V31</th>\n",
       "      <th>V32</th>\n",
       "      <th>V33</th>\n",
       "      <th>V34</th>\n",
       "      <th>V35</th>\n",
       "      <th>V36</th>\n",
       "      <th>V37</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.566</td>\n",
       "      <td>0.016</td>\n",
       "      <td>-0.143</td>\n",
       "      <td>0.407</td>\n",
       "      <td>0.452</td>\n",
       "      <td>-0.901</td>\n",
       "      <td>-1.812</td>\n",
       "      <td>-2.360</td>\n",
       "      <td>-0.436</td>\n",
       "      <td>-2.114</td>\n",
       "      <td>...</td>\n",
       "      <td>0.136</td>\n",
       "      <td>0.109</td>\n",
       "      <td>-0.615</td>\n",
       "      <td>0.327</td>\n",
       "      <td>-4.627</td>\n",
       "      <td>-4.789</td>\n",
       "      <td>-5.101</td>\n",
       "      <td>-2.608</td>\n",
       "      <td>-3.508</td>\n",
       "      <td>0.175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.968</td>\n",
       "      <td>0.437</td>\n",
       "      <td>0.066</td>\n",
       "      <td>0.566</td>\n",
       "      <td>0.194</td>\n",
       "      <td>-0.893</td>\n",
       "      <td>-1.566</td>\n",
       "      <td>-2.360</td>\n",
       "      <td>0.332</td>\n",
       "      <td>-2.114</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.128</td>\n",
       "      <td>0.124</td>\n",
       "      <td>0.032</td>\n",
       "      <td>0.600</td>\n",
       "      <td>-0.843</td>\n",
       "      <td>0.160</td>\n",
       "      <td>0.364</td>\n",
       "      <td>-0.335</td>\n",
       "      <td>-0.730</td>\n",
       "      <td>0.676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.013</td>\n",
       "      <td>0.568</td>\n",
       "      <td>0.235</td>\n",
       "      <td>0.370</td>\n",
       "      <td>0.112</td>\n",
       "      <td>-0.797</td>\n",
       "      <td>-1.367</td>\n",
       "      <td>-2.360</td>\n",
       "      <td>0.396</td>\n",
       "      <td>-2.114</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.009</td>\n",
       "      <td>0.361</td>\n",
       "      <td>0.277</td>\n",
       "      <td>-0.116</td>\n",
       "      <td>-0.843</td>\n",
       "      <td>0.160</td>\n",
       "      <td>0.364</td>\n",
       "      <td>0.765</td>\n",
       "      <td>-0.589</td>\n",
       "      <td>0.633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.733</td>\n",
       "      <td>0.368</td>\n",
       "      <td>0.283</td>\n",
       "      <td>0.165</td>\n",
       "      <td>0.599</td>\n",
       "      <td>-0.679</td>\n",
       "      <td>-1.200</td>\n",
       "      <td>-2.086</td>\n",
       "      <td>0.403</td>\n",
       "      <td>-2.114</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.417</td>\n",
       "      <td>0.279</td>\n",
       "      <td>0.603</td>\n",
       "      <td>-0.843</td>\n",
       "      <td>-0.065</td>\n",
       "      <td>0.364</td>\n",
       "      <td>0.333</td>\n",
       "      <td>-0.112</td>\n",
       "      <td>0.206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.684</td>\n",
       "      <td>0.638</td>\n",
       "      <td>0.260</td>\n",
       "      <td>0.209</td>\n",
       "      <td>0.337</td>\n",
       "      <td>-0.454</td>\n",
       "      <td>-1.073</td>\n",
       "      <td>-2.086</td>\n",
       "      <td>0.314</td>\n",
       "      <td>-2.114</td>\n",
       "      <td>...</td>\n",
       "      <td>0.183</td>\n",
       "      <td>1.078</td>\n",
       "      <td>0.328</td>\n",
       "      <td>0.418</td>\n",
       "      <td>-0.843</td>\n",
       "      <td>-0.215</td>\n",
       "      <td>0.364</td>\n",
       "      <td>-0.280</td>\n",
       "      <td>-0.028</td>\n",
       "      <td>0.384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2883</th>\n",
       "      <td>0.190</td>\n",
       "      <td>-0.025</td>\n",
       "      <td>-0.138</td>\n",
       "      <td>0.161</td>\n",
       "      <td>0.600</td>\n",
       "      <td>-0.212</td>\n",
       "      <td>0.757</td>\n",
       "      <td>0.584</td>\n",
       "      <td>-0.026</td>\n",
       "      <td>0.904</td>\n",
       "      <td>...</td>\n",
       "      <td>0.128</td>\n",
       "      <td>-0.208</td>\n",
       "      <td>0.809</td>\n",
       "      <td>-0.173</td>\n",
       "      <td>0.247</td>\n",
       "      <td>-0.027</td>\n",
       "      <td>-0.349</td>\n",
       "      <td>0.576</td>\n",
       "      <td>0.686</td>\n",
       "      <td>0.235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2884</th>\n",
       "      <td>0.507</td>\n",
       "      <td>0.557</td>\n",
       "      <td>0.296</td>\n",
       "      <td>0.183</td>\n",
       "      <td>0.530</td>\n",
       "      <td>-0.237</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.584</td>\n",
       "      <td>0.537</td>\n",
       "      <td>0.904</td>\n",
       "      <td>...</td>\n",
       "      <td>0.291</td>\n",
       "      <td>-0.287</td>\n",
       "      <td>0.465</td>\n",
       "      <td>-0.310</td>\n",
       "      <td>0.763</td>\n",
       "      <td>0.498</td>\n",
       "      <td>-0.349</td>\n",
       "      <td>-0.615</td>\n",
       "      <td>-0.380</td>\n",
       "      <td>1.042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2885</th>\n",
       "      <td>-0.394</td>\n",
       "      <td>-0.721</td>\n",
       "      <td>-0.485</td>\n",
       "      <td>0.084</td>\n",
       "      <td>0.136</td>\n",
       "      <td>0.034</td>\n",
       "      <td>0.655</td>\n",
       "      <td>0.614</td>\n",
       "      <td>-0.818</td>\n",
       "      <td>0.904</td>\n",
       "      <td>...</td>\n",
       "      <td>0.291</td>\n",
       "      <td>-0.179</td>\n",
       "      <td>0.268</td>\n",
       "      <td>0.552</td>\n",
       "      <td>0.763</td>\n",
       "      <td>0.498</td>\n",
       "      <td>-0.349</td>\n",
       "      <td>0.951</td>\n",
       "      <td>0.748</td>\n",
       "      <td>0.005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2886</th>\n",
       "      <td>-0.219</td>\n",
       "      <td>-0.282</td>\n",
       "      <td>-0.344</td>\n",
       "      <td>-0.049</td>\n",
       "      <td>0.449</td>\n",
       "      <td>-0.140</td>\n",
       "      <td>0.560</td>\n",
       "      <td>0.583</td>\n",
       "      <td>-0.596</td>\n",
       "      <td>0.904</td>\n",
       "      <td>...</td>\n",
       "      <td>0.216</td>\n",
       "      <td>1.061</td>\n",
       "      <td>-0.051</td>\n",
       "      <td>1.023</td>\n",
       "      <td>0.878</td>\n",
       "      <td>0.610</td>\n",
       "      <td>-0.230</td>\n",
       "      <td>-0.301</td>\n",
       "      <td>0.555</td>\n",
       "      <td>0.350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2887</th>\n",
       "      <td>0.368</td>\n",
       "      <td>0.380</td>\n",
       "      <td>-0.225</td>\n",
       "      <td>-0.049</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.092</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.551</td>\n",
       "      <td>0.244</td>\n",
       "      <td>0.904</td>\n",
       "      <td>...</td>\n",
       "      <td>0.047</td>\n",
       "      <td>0.057</td>\n",
       "      <td>-0.042</td>\n",
       "      <td>0.847</td>\n",
       "      <td>0.534</td>\n",
       "      <td>-0.009</td>\n",
       "      <td>-0.190</td>\n",
       "      <td>-0.567</td>\n",
       "      <td>0.388</td>\n",
       "      <td>0.417</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2888 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         V0     V1     V2     V3     V4     V5     V6     V7     V8     V9  \\\n",
       "0     0.566  0.016 -0.143  0.407  0.452 -0.901 -1.812 -2.360 -0.436 -2.114   \n",
       "1     0.968  0.437  0.066  0.566  0.194 -0.893 -1.566 -2.360  0.332 -2.114   \n",
       "2     1.013  0.568  0.235  0.370  0.112 -0.797 -1.367 -2.360  0.396 -2.114   \n",
       "3     0.733  0.368  0.283  0.165  0.599 -0.679 -1.200 -2.086  0.403 -2.114   \n",
       "4     0.684  0.638  0.260  0.209  0.337 -0.454 -1.073 -2.086  0.314 -2.114   \n",
       "...     ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "2883  0.190 -0.025 -0.138  0.161  0.600 -0.212  0.757  0.584 -0.026  0.904   \n",
       "2884  0.507  0.557  0.296  0.183  0.530 -0.237  0.749  0.584  0.537  0.904   \n",
       "2885 -0.394 -0.721 -0.485  0.084  0.136  0.034  0.655  0.614 -0.818  0.904   \n",
       "2886 -0.219 -0.282 -0.344 -0.049  0.449 -0.140  0.560  0.583 -0.596  0.904   \n",
       "2887  0.368  0.380 -0.225 -0.049  0.379  0.092  0.550  0.551  0.244  0.904   \n",
       "\n",
       "      ...    V29    V30    V31    V32    V33    V34    V35    V36    V37  \\\n",
       "0     ...  0.136  0.109 -0.615  0.327 -4.627 -4.789 -5.101 -2.608 -3.508   \n",
       "1     ... -0.128  0.124  0.032  0.600 -0.843  0.160  0.364 -0.335 -0.730   \n",
       "2     ... -0.009  0.361  0.277 -0.116 -0.843  0.160  0.364  0.765 -0.589   \n",
       "3     ...  0.015  0.417  0.279  0.603 -0.843 -0.065  0.364  0.333 -0.112   \n",
       "4     ...  0.183  1.078  0.328  0.418 -0.843 -0.215  0.364 -0.280 -0.028   \n",
       "...   ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "2883  ...  0.128 -0.208  0.809 -0.173  0.247 -0.027 -0.349  0.576  0.686   \n",
       "2884  ...  0.291 -0.287  0.465 -0.310  0.763  0.498 -0.349 -0.615 -0.380   \n",
       "2885  ...  0.291 -0.179  0.268  0.552  0.763  0.498 -0.349  0.951  0.748   \n",
       "2886  ...  0.216  1.061 -0.051  1.023  0.878  0.610 -0.230 -0.301  0.555   \n",
       "2887  ...  0.047  0.057 -0.042  0.847  0.534 -0.009 -0.190 -0.567  0.388   \n",
       "\n",
       "      target  \n",
       "0      0.175  \n",
       "1      0.676  \n",
       "2      0.633  \n",
       "3      0.206  \n",
       "4      0.384  \n",
       "...      ...  \n",
       "2883   0.235  \n",
       "2884   1.042  \n",
       "2885   0.005  \n",
       "2886   0.350  \n",
       "2887   0.417  \n",
       "\n",
       "[2888 rows x 39 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#划分验证集\n",
    "xtrain,xtest,ytrain,ytest = train_test_split(train_trainf,train_traint,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE_fake: 0.1241285601036111\n",
      "MSE_true: 0.13363855962001847\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9807046731347451, 0.8477377565123823, 0.8667129598462455)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#随机森林\n",
    "m1 = RandomForestRegressor()\n",
    "m1.fit(xtrain,ytrain)\n",
    "print('MSE_fake:',mean_squared_error(ytest,m1.predict(xtest)))\n",
    "print('MSE_true:',mean_squared_error(test_target,m1.predict(test_feature[pre_feature])))\n",
    "m1.score(xtrain,ytrain),m1.score(xtest,ytest),m1.score(test_feature[pre_feature],test_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, ccp_alpha=0.0, criterion='mse',\n",
       "                      max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "                      max_samples=None, min_impurity_decrease=0.0,\n",
       "                      min_impurity_split=None, min_samples_leaf=1,\n",
       "                      min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "                      n_estimators=100, n_jobs=None, oob_score=False,\n",
       "                      random_state=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE_fake: 0.11152855825842213\n",
      "MSE_true: 0.1284936729199363\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9324187948431065, 0.86319354321687, 0.8718443135672838)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#GBR\n",
    "m2 = GradientBoostingRegressor()\n",
    "m2.fit(xtrain,ytrain)\n",
    "print('MSE_fake:',mean_squared_error(ytest,m2.predict(xtest)))\n",
    "print('MSE_true:',mean_squared_error(test_target,m2.predict(test_feature[pre_feature])))\n",
    "m2.score(xtrain,ytrain),m2.score(xtest,ytest),m2.score(test_feature[pre_feature],test_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, ccp_alpha=0.0, criterion='friedman_mse',\n",
       "                          init=None, learning_rate=0.1, loss='ls', max_depth=3,\n",
       "                          max_features=None, max_leaf_nodes=None,\n",
       "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                          min_samples_leaf=1, min_samples_split=2,\n",
       "                          min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                          n_iter_no_change=None, presort='deprecated',\n",
       "                          random_state=None, subsample=1.0, tol=0.0001,\n",
       "                          validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE_fake: 0.14602068343792954\n",
      "MSE_true: 0.14400320948877368\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9998968061205673, 0.8495882523594338, 0.8508959746099177)"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#XGboost\n",
    "m4 = XGBRegressor()\n",
    "m4.fit(xtrain,ytrain)\n",
    "print('MSE_fake:',mean_squared_error(ytest,m4.predict(xtest)))\n",
    "print('MSE_true:',mean_squared_error(test_target,m4.predict(test_feature[pre_feature])))\n",
    "m4.score(xtrain,ytrain),m4.score(xtest,ytest),m4.score(test_feature[pre_feature],test_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE_fake: 0.15212595973007437\n",
      "MSE_true: 0.14759997698818106\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9291552039931625, 0.8432993811166116, 0.8471718040552632)"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#XGBRF\n",
    "m5 = XGBRFRegressor()\n",
    "m5.fit(xtrain,ytrain)\n",
    "print('MSE_fake:',mean_squared_error(ytest,m5.predict(xtest)))\n",
    "print('MSE_true:',mean_squared_error(test_target,m5.predict(test_feature[pre_feature])))\n",
    "m5.score(xtrain,ytrain),m5.score(xtest,ytest),m5.score(test_feature[pre_feature],test_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\my_project\\mine_happiness\\lib\\site-packages\\sklearn\\ensemble\\_gb.py:1342: FutureWarning: The parameter 'presort' is deprecated and has no effect. It will be removed in v0.24. You can suppress this warning by not passing any value to the 'presort' parameter. We also recommend using HistGradientBoosting models instead.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE_fake: 0.11760276154546842\n",
      "MSE_true: 0.120764220275785\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9859941419478747, 0.8788607444168699, 0.8749581246825066)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#GBR\n",
    "m_lin =  GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
    "                                  learning_rate=0.03, loss='huber', max_depth=15,\n",
    "                                  max_features='sqrt', max_leaf_nodes=None,\n",
    "                                  min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "                                  min_samples_leaf=10, min_samples_split=40,\n",
    "                                  min_weight_fraction_leaf=0.0, n_estimators=300,\n",
    "                                  presort='auto', random_state=10, subsample=0.8, verbose=0,\n",
    "                                  warm_start=False)\n",
    "m_lin.fit(xtrain,ytrain)\n",
    "print('MSE_fake:',mean_squared_error(ytest,m_lin.predict(xtest)))\n",
    "print('MSE_true:',mean_squared_error(test_target,m_lin.predict(test_feature[pre_feature])))\n",
    "m_lin.score(xtrain,ytrain),m_lin.score(xtest,ytest),m_lin.score(test_feature[pre_feature],test_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE_fake: 0.143312704080843\n",
      "MSE_true: 0.13202268597525496\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8835870502758365, 0.852377664777484, 0.8633008667542518)"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SGD\n",
    "随机梯度回归 = SGDRegressor()\n",
    "随机梯度回归.fit(xtrain,ytrain)\n",
    "print('MSE_fake:',mean_squared_error(ytest,随机梯度回归.predict(xtest)))\n",
    "print('MSE_true:',mean_squared_error(test_target,随机梯度回归.predict(test_feature[pre_feature])))\n",
    "随机梯度回归.score(xtrain,ytrain),随机梯度回归.score(xtest,ytest),随机梯度回归.score(test_feature[pre_feature],test_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\my_project\\mine_happiness\\lib\\site-packages\\sklearn\\ensemble\\_gb.py:1342: FutureWarning: The parameter 'presort' is deprecated and has no effect. It will be removed in v0.24. You can suppress this warning by not passing any value to the 'presort' parameter. We also recommend using HistGradientBoosting models instead.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE: 0.11947829878118113\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.986314758813013, 0.8762895955008428)"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#训练集和验证集合并训练\n",
    "m_lin.fit(train_trainf,train_traint)\n",
    "print('MSE:',mean_squared_error(test_target,m_lin.predict(test_feature[pre_feature])))\n",
    "m_lin.score(train_trainf,train_traint),m_lin.score(test_feature[pre_feature],test_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\my_project\\mine_happiness\\lib\\site-packages\\sklearn\\ensemble\\_gb.py:1342: FutureWarning: The parameter 'presort' is deprecated and has no effect. It will be removed in v0.24. You can suppress this warning by not passing any value to the 'presort' parameter. We also recommend using HistGradientBoosting models instead.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, ccp_alpha=0.0, criterion='friedman_mse',\n",
       "                          init=None, learning_rate=0.03, loss='huber',\n",
       "                          max_depth=15, max_features='sqrt',\n",
       "                          max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "                          min_impurity_split=None, min_samples_leaf=10,\n",
       "                          min_samples_split=40, min_weight_fraction_leaf=0.0,\n",
       "                          n_estimators=300, n_iter_no_change=None,\n",
       "                          presort='auto', random_state=10, subsample=0.8,\n",
       "                          tol=0.0001, validation_fraction=0.1, verbose=0,\n",
       "                          warm_start=False)"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_lin.fit(final_train,final_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V0</th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V10</th>\n",
       "      <th>V13</th>\n",
       "      <th>V15</th>\n",
       "      <th>V18</th>\n",
       "      <th>V19</th>\n",
       "      <th>V24</th>\n",
       "      <th>V29</th>\n",
       "      <th>V30</th>\n",
       "      <th>V31</th>\n",
       "      <th>V36</th>\n",
       "      <th>V37</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.368</td>\n",
       "      <td>0.380</td>\n",
       "      <td>-0.225</td>\n",
       "      <td>-0.049</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.551</td>\n",
       "      <td>0.244</td>\n",
       "      <td>-0.419</td>\n",
       "      <td>-0.114</td>\n",
       "      <td>0.239</td>\n",
       "      <td>0.247</td>\n",
       "      <td>0.899</td>\n",
       "      <td>-1.314</td>\n",
       "      <td>0.047</td>\n",
       "      <td>0.057</td>\n",
       "      <td>-0.042</td>\n",
       "      <td>-0.567</td>\n",
       "      <td>0.388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.148</td>\n",
       "      <td>0.489</td>\n",
       "      <td>-0.247</td>\n",
       "      <td>-0.049</td>\n",
       "      <td>0.487</td>\n",
       "      <td>0.493</td>\n",
       "      <td>-0.127</td>\n",
       "      <td>-0.403</td>\n",
       "      <td>0.653</td>\n",
       "      <td>-0.113</td>\n",
       "      <td>0.073</td>\n",
       "      <td>1.168</td>\n",
       "      <td>-1.310</td>\n",
       "      <td>0.047</td>\n",
       "      <td>0.560</td>\n",
       "      <td>0.176</td>\n",
       "      <td>-0.294</td>\n",
       "      <td>0.104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.166</td>\n",
       "      <td>-0.062</td>\n",
       "      <td>-0.311</td>\n",
       "      <td>0.046</td>\n",
       "      <td>0.485</td>\n",
       "      <td>0.493</td>\n",
       "      <td>-0.227</td>\n",
       "      <td>0.330</td>\n",
       "      <td>0.398</td>\n",
       "      <td>-0.192</td>\n",
       "      <td>0.070</td>\n",
       "      <td>0.980</td>\n",
       "      <td>-1.310</td>\n",
       "      <td>-0.398</td>\n",
       "      <td>0.101</td>\n",
       "      <td>0.199</td>\n",
       "      <td>0.373</td>\n",
       "      <td>0.569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.102</td>\n",
       "      <td>0.294</td>\n",
       "      <td>-0.259</td>\n",
       "      <td>0.051</td>\n",
       "      <td>0.474</td>\n",
       "      <td>0.504</td>\n",
       "      <td>0.010</td>\n",
       "      <td>-0.431</td>\n",
       "      <td>-0.340</td>\n",
       "      <td>-0.590</td>\n",
       "      <td>0.078</td>\n",
       "      <td>1.070</td>\n",
       "      <td>0.234</td>\n",
       "      <td>-0.398</td>\n",
       "      <td>1.007</td>\n",
       "      <td>0.137</td>\n",
       "      <td>-0.666</td>\n",
       "      <td>0.391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.300</td>\n",
       "      <td>0.428</td>\n",
       "      <td>0.208</td>\n",
       "      <td>0.051</td>\n",
       "      <td>0.408</td>\n",
       "      <td>0.497</td>\n",
       "      <td>0.155</td>\n",
       "      <td>-0.162</td>\n",
       "      <td>0.611</td>\n",
       "      <td>-0.927</td>\n",
       "      <td>0.080</td>\n",
       "      <td>1.238</td>\n",
       "      <td>0.237</td>\n",
       "      <td>-0.776</td>\n",
       "      <td>0.291</td>\n",
       "      <td>0.370</td>\n",
       "      <td>-0.140</td>\n",
       "      <td>-0.497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1920</th>\n",
       "      <td>-1.362</td>\n",
       "      <td>-1.553</td>\n",
       "      <td>-3.096</td>\n",
       "      <td>-0.444</td>\n",
       "      <td>-4.854</td>\n",
       "      <td>-5.331</td>\n",
       "      <td>-4.074</td>\n",
       "      <td>-2.551</td>\n",
       "      <td>0.928</td>\n",
       "      <td>0.867</td>\n",
       "      <td>-3.573</td>\n",
       "      <td>0.107</td>\n",
       "      <td>-0.630</td>\n",
       "      <td>0.171</td>\n",
       "      <td>-4.488</td>\n",
       "      <td>-5.793</td>\n",
       "      <td>-2.564</td>\n",
       "      <td>0.597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1921</th>\n",
       "      <td>-2.698</td>\n",
       "      <td>-3.452</td>\n",
       "      <td>-3.620</td>\n",
       "      <td>-1.066</td>\n",
       "      <td>-4.927</td>\n",
       "      <td>-5.103</td>\n",
       "      <td>-4.393</td>\n",
       "      <td>-2.525</td>\n",
       "      <td>1.871</td>\n",
       "      <td>1.135</td>\n",
       "      <td>-0.965</td>\n",
       "      <td>0.193</td>\n",
       "      <td>-0.204</td>\n",
       "      <td>1.297</td>\n",
       "      <td>-0.613</td>\n",
       "      <td>-7.698</td>\n",
       "      <td>-2.564</td>\n",
       "      <td>1.215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1922</th>\n",
       "      <td>-2.615</td>\n",
       "      <td>-3.564</td>\n",
       "      <td>-3.402</td>\n",
       "      <td>-0.422</td>\n",
       "      <td>-4.223</td>\n",
       "      <td>-4.315</td>\n",
       "      <td>-5.196</td>\n",
       "      <td>-2.529</td>\n",
       "      <td>1.976</td>\n",
       "      <td>0.504</td>\n",
       "      <td>-1.568</td>\n",
       "      <td>0.301</td>\n",
       "      <td>1.057</td>\n",
       "      <td>0.552</td>\n",
       "      <td>0.125</td>\n",
       "      <td>-6.111</td>\n",
       "      <td>-2.544</td>\n",
       "      <td>1.612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1923</th>\n",
       "      <td>-2.661</td>\n",
       "      <td>-3.646</td>\n",
       "      <td>-3.271</td>\n",
       "      <td>-0.699</td>\n",
       "      <td>-3.716</td>\n",
       "      <td>-3.809</td>\n",
       "      <td>-4.735</td>\n",
       "      <td>-2.560</td>\n",
       "      <td>1.520</td>\n",
       "      <td>0.206</td>\n",
       "      <td>-1.282</td>\n",
       "      <td>-0.036</td>\n",
       "      <td>0.800</td>\n",
       "      <td>0.318</td>\n",
       "      <td>1.086</td>\n",
       "      <td>-5.268</td>\n",
       "      <td>-2.549</td>\n",
       "      <td>1.431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1924</th>\n",
       "      <td>-2.321</td>\n",
       "      <td>-3.037</td>\n",
       "      <td>-3.214</td>\n",
       "      <td>-1.594</td>\n",
       "      <td>-3.616</td>\n",
       "      <td>-3.747</td>\n",
       "      <td>-4.368</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.847</td>\n",
       "      <td>0.206</td>\n",
       "      <td>-1.213</td>\n",
       "      <td>0.592</td>\n",
       "      <td>0.799</td>\n",
       "      <td>0.323</td>\n",
       "      <td>-0.774</td>\n",
       "      <td>-5.211</td>\n",
       "      <td>-1.123</td>\n",
       "      <td>1.988</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1925 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         V0     V1     V2     V3     V6     V7     V8    V10    V13    V15  \\\n",
       "0     0.368  0.380 -0.225 -0.049  0.550  0.551  0.244 -0.419 -0.114  0.239   \n",
       "1     0.148  0.489 -0.247 -0.049  0.487  0.493 -0.127 -0.403  0.653 -0.113   \n",
       "2    -0.166 -0.062 -0.311  0.046  0.485  0.493 -0.227  0.330  0.398 -0.192   \n",
       "3     0.102  0.294 -0.259  0.051  0.474  0.504  0.010 -0.431 -0.340 -0.590   \n",
       "4     0.300  0.428  0.208  0.051  0.408  0.497  0.155 -0.162  0.611 -0.927   \n",
       "...     ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "1920 -1.362 -1.553 -3.096 -0.444 -4.854 -5.331 -4.074 -2.551  0.928  0.867   \n",
       "1921 -2.698 -3.452 -3.620 -1.066 -4.927 -5.103 -4.393 -2.525  1.871  1.135   \n",
       "1922 -2.615 -3.564 -3.402 -0.422 -4.223 -4.315 -5.196 -2.529  1.976  0.504   \n",
       "1923 -2.661 -3.646 -3.271 -0.699 -3.716 -3.809 -4.735 -2.560  1.520  0.206   \n",
       "1924 -2.321 -3.037 -3.214 -1.594 -3.616 -3.747 -4.368  0.056  0.847  0.206   \n",
       "\n",
       "        V18    V19    V24    V29    V30    V31    V36    V37  \n",
       "0     0.247  0.899 -1.314  0.047  0.057 -0.042 -0.567  0.388  \n",
       "1     0.073  1.168 -1.310  0.047  0.560  0.176 -0.294  0.104  \n",
       "2     0.070  0.980 -1.310 -0.398  0.101  0.199  0.373  0.569  \n",
       "3     0.078  1.070  0.234 -0.398  1.007  0.137 -0.666  0.391  \n",
       "4     0.080  1.238  0.237 -0.776  0.291  0.370 -0.140 -0.497  \n",
       "...     ...    ...    ...    ...    ...    ...    ...    ...  \n",
       "1920 -3.573  0.107 -0.630  0.171 -4.488 -5.793 -2.564  0.597  \n",
       "1921 -0.965  0.193 -0.204  1.297 -0.613 -7.698 -2.564  1.215  \n",
       "1922 -1.568  0.301  1.057  0.552  0.125 -6.111 -2.544  1.612  \n",
       "1923 -1.282 -0.036  0.800  0.318  1.086 -5.268 -2.549  1.431  \n",
       "1924 -1.213  0.592  0.799  0.323 -0.774 -5.211 -1.123  1.988  \n",
       "\n",
       "[1925 rows x 18 columns]"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_test = test_data[pre_feature]\n",
    "final_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1925,)"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict18 = m_lin.predict(final_test)\n",
    "predict18.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('.//result//predict18.txt',predict18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
